import os
import numpy as np
import pandas as pd
from glob import glob
from pathlib import Path
import shutil
from sklearn.model_selection import train_test_split
# image
import cv2
from skimage.io import imread
# TensorFlow
import tensorflow as tf
from tensorflow_examples.models.pix2pix import pix2pix
from tensorflow.keras import layers, models
# Visualisation libraries
## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output
## progressbar
from tqdm import tqdm
## plotly
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px
## maps
import folium
from folium import plugins
from folium.plugins import HeatMap
## seaborn
import seaborn as sns
## matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
from matplotlib.font_manager import FontProperties
import matplotlib.colors as mcolors
from matplotlib.colors import LinearSegmentedColormap
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from matplotlib import cm
plt.style.use('seaborn-whitegrid')
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['text.color'] = 'k'
%matplotlib inline
import warnings
warnings.filterwarnings("ignore")
Competitions like LUNA (http://luna16.grand-challenge.org) and the Kaggle Data Science Bowl 2017 (https://www.kaggle.com/c/data-science-bowl-2017) involve processing and trying to find lesions in CT images of the lungs. In order to find disease in these images well, it is important to first find the lungs well. This dataset is a collection of 2D and 3D images with manually segmented lungs.
Start_Path = 'lung_CT'
# Directory
def Path_Tree(startpath, Extension, sep = ' ' * 3, n = 5):
Folders_with_Images = []
C = ['Red', 'Green', 'Magenta', 'Cyan']*len(os.listdir(startpath))
BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN,
'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
for root, _, files in os.walk(startpath):
level = root.replace(startpath, '').count(os.sep)
if level >0:
indent = sep* (level)+ '└──'
print(indent + BACK[C[level]] + Fore.BLACK + Style.NORMAL + os.path.basename(root) + Style.RESET_ALL)
else:
title = os.path.basename(root)
print(Style.RESET_ALL + Fore.BLUE + Style.NORMAL + '=' * (len(title) +1) + Style.RESET_ALL)
print(Back.BLACK + Fore.CYAN + Style.NORMAL + title + Style.RESET_ALL)
print(Style.RESET_ALL + Fore.BLUE + Style.NORMAL + '=' * (len(title) +1)+ Style.RESET_ALL)
subindent = ' ' * 4 * (level + 1)
for file in files[:1]:
if file.endswith(Extension):
Folders_with_Images.append(root)
List = os.listdir(root)
print(level* sep, Fore.BLUE + Style.NORMAL +
'%i %s files: ' % (len(List), List[0].split('.')[-1].upper()) + Style.RESET_ALL +
'%s'%', '.join(List[:n]) + ', ...')
return Folders_with_Images
_ = Folders_with_Images = Path_Tree(Start_Path, '.tif', n = 3)
======== lung_CT ======== └──2d_images 267 TIF files: ID_0000_Z_0142.tif, ID_0001_Z_0146.tif, ID_0002_Z_0162.tif, ... └──2d_masks 267 TIF files: ID_0000_Z_0142.tif, ID_0001_Z_0146.tif, ID_0002_Z_0162.tif, ...
We can convert the TIFF images to JPEG files and copy them into a new directory.
def Tif2JPG(src, dst):
'''
Converts tiff images to jpeg images.
src: source folder
dst: destination folder
'''
if os.path.exists(dst):
shutil.rmtree(dst)
folders = list({x.parent.stem for x in Path(src).glob('**/*.tif')})
new_folders = [x + '_mod' for x in folders]
for subfolder in new_folders:
if not os.path.exists(os.path.join(dst, subfolder)):
os.makedirs(os.path.join(dst, subfolder))
for folder, new_folder in zip(folders, new_folders):
c = 0
for x in Path(src).glob('**/'+ folder +'/*.tif'):
cv2.imwrite(os.path.join(dst, new_folder, x.stem + '.jpg'),
imread(x), [int(cv2.IMWRITE_JPEG_QUALITY), 100])
c+=1
print(Fore.BLUE + 'Converted %i tif images to jpg images: from %s to %s' %\
(c, folder, new_folder) + Style.RESET_ALL)
NewPath = os.path.join(Start_Path, 'train_jpg')
Tif2JPG(src = Start_Path, dst = NewPath)
Converted 267 tif images to jpg images: from 2d_masks to 2d_masks_mod Converted 267 tif images to jpg images: from 2d_images to 2d_images_mod
# Dataset
Data = pd.read_csv(os.path.join(Start_Path,'lung_stats.csv'))
Data['img_id'] = Data['img_id'].map(lambda x: x.replace('tif','jpg'))
Path_dict = {(x.parent.stem, os.path.basename(x)): x for x in Path(Start_Path).glob('**/*.jpg')}
Data['Image_Path'] = Data['img_id'].map(lambda x: Path_dict[('2d_images_mod', x)])
Data['Mask_Path'] = Data['img_id'].map(lambda x: Path_dict[('2d_masks_mod', x)])
del Path_dict
#
def Header(Text, L = 100, C = 'Blue', T = 'White'):
BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN, 'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
print(BACK[C] + FORE[T] + Style.NORMAL + Text + Style.RESET_ALL + ' ' + FORE[C] +
Style.NORMAL + (L- len(Text) - 1)*'=' + Style.RESET_ALL)
def Line(L=100, C = 'Blue'):
FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
print(FORE[C] + Style.NORMAL + L*'=' + Style.RESET_ALL)
Header('A Sample of the Dataframe')
display(Data.sample(10))
display(pd.DataFrame({'Number of Instances':[Data.shape[0]], 'Number of Attributes':[Data.shape[1]]}).style.hide_index())
def DataPlotX(Inp, Title = None, H = None):
data_info = Inp.dtypes.astype(str).to_frame(name='Data Type')
Temp = Inp.isnull().sum().to_frame(name = 'Number of NaN Values')
data_info = data_info.join(Temp, how='outer')
data_info ['Size'] = Inp.shape[0]
data_info['Percentage'] = 100 - np.round(100*(data_info['Number of NaN Values']/Inp.shape[0]),2)
data_info = data_info.reset_index(drop = False).rename(columns = {'index':'Features'})
#
fig = px.bar(data_info, y= 'Features', x= 'Percentage', color = 'Data Type', text = 'Percentage',
color_discrete_sequence = ['PaleGreen', 'LightCyan', 'PeachPuff', 'Pink', 'Plum'],
hover_data = data_info.columns)
fig.update_layout(plot_bgcolor= 'white', legend=dict(x=1.01, y=.5, traceorder="normal",
bordercolor="DarkGray", borderwidth=1))
if not H == None:
fig.update_layout(height = H)
fig.update_traces(texttemplate= 10*' ' + '%%{text}', textposition='inside')
fig.update_traces(marker_line_color= 'Black', marker_line_width=1., opacity=1)
if not Title == None:
fig.update_layout(title={'text': '<b>' + Title + '<b>', 'x':0.5,
'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
fig.show()
return data_info
data_info = DataPlotX(Data, Title = 'Lungs CT Data', H = 350)
A Sample of the Dataframe ==========================================================================
| img_id | lung_area_px | lung_area_mm2 | lung_volume_fraction | lung_mean_hu | lung_pd95_hu | lung_pd05_hu | Image_Path | Mask_Path | |
|---|---|---|---|---|---|---|---|---|---|
| 217 | ID_0217_Z_0078.jpg | 68454 | 48131.718750 | 0.261131 | -644.736495 | -294.65 | -809.0 | lung_CT\train_jpg\2d_images_mod\ID_0217_Z_0078... | lung_CT\train_jpg\2d_masks_mod\ID_0217_Z_0078.jpg |
| 191 | ID_0191_Z_0140.jpg | 35627 | 25050.234375 | 0.135906 | -597.434193 | -237.00 | -794.0 | lung_CT\train_jpg\2d_images_mod\ID_0191_Z_0140... | lung_CT\train_jpg\2d_masks_mod\ID_0191_Z_0140.jpg |
| 9 | ID_0009_Z_0114.jpg | 62030 | 43614.843750 | 0.236626 | -661.701290 | -345.00 | -836.0 | lung_CT\train_jpg\2d_images_mod\ID_0009_Z_0114... | lung_CT\train_jpg\2d_masks_mod\ID_0009_Z_0114.jpg |
| 209 | ID_0209_Z_0267.jpg | 75884 | 53355.937500 | 0.289474 | -707.899280 | -472.00 | -860.0 | lung_CT\train_jpg\2d_images_mod\ID_0209_Z_0267... | lung_CT\train_jpg\2d_masks_mod\ID_0209_Z_0267.jpg |
| 108 | ID_0108_Z_0259.jpg | 73999 | 52030.546875 | 0.282284 | -675.109001 | -445.00 | -809.0 | lung_CT\train_jpg\2d_images_mod\ID_0108_Z_0259... | lung_CT\train_jpg\2d_masks_mod\ID_0108_Z_0259.jpg |
| 68 | ID_0068_Z_0055.jpg | 54531 | 38342.109375 | 0.208019 | -587.960279 | -129.00 | -801.0 | lung_CT\train_jpg\2d_images_mod\ID_0068_Z_0055... | lung_CT\train_jpg\2d_masks_mod\ID_0068_Z_0055.jpg |
| 214 | ID_0214_Z_0120.jpg | 44961 | 31613.203125 | 0.171513 | -593.887814 | -159.00 | -868.0 | lung_CT\train_jpg\2d_images_mod\ID_0214_Z_0120... | lung_CT\train_jpg\2d_masks_mod\ID_0214_Z_0120.jpg |
| 158 | ID_0158_Z_0164.jpg | 43895 | 30863.671875 | 0.167446 | -645.264768 | -298.70 | -812.0 | lung_CT\train_jpg\2d_images_mod\ID_0158_Z_0164... | lung_CT\train_jpg\2d_masks_mod\ID_0158_Z_0164.jpg |
| 89 | ID_0089_Z_0341.jpg | 81734 | 57469.218750 | 0.311790 | -682.772371 | -387.00 | -838.0 | lung_CT\train_jpg\2d_images_mod\ID_0089_Z_0341... | lung_CT\train_jpg\2d_masks_mod\ID_0089_Z_0341.jpg |
| 43 | ID_0043_Z_0097.jpg | 58816 | 41355.000000 | 0.224365 | -626.435885 | -308.75 | -787.0 | lung_CT\train_jpg\2d_images_mod\ID_0043_Z_0097... | lung_CT\train_jpg\2d_masks_mod\ID_0043_Z_0097.jpg |
| Number of Instances | Number of Attributes |
|---|---|
| 267 | 9 |
fig, ax = plt.subplots(4, 4 , figsize = (12, 12))
_ = fig.suptitle('A Sample of Train Dataset', fontweight='bold', fontsize = 18)
ax = ax.ravel()
for i, row in Data.sample(int(len(ax)/2)).reset_index(drop =True).iterrows():
_ = ax[2*i].imshow(imread(row['Image_Path']), cmap='bone')
_ = ax[2*i].set_title('CT Image', fontweight='bold', fontsize = 12)
_ = ax[2*i + 1].imshow(imread(row['Mask_Path']), cmap='binary')
_ = ax[2*i + 1].set_title('Image Mask', fontweight='bold', fontsize = 12)
for i in range(len(ax)):
_ = ax[i].axis("off")
_ = ax[i].set_aspect(1)
fig.tight_layout()
# Image size for training model
Img_Size = [64, 64]
def normalize(Img, Img_msk):
Img = tf.cast(Img, tf.float32) / 255.0
Img_msk -= 1
return Img, Img_msk
@tf.function
def DataAugmentation(Img, Img_msk, Normalize = False):
if tf.random.uniform(()) > 0.5:
Img = tf.image.flip_left_right(Img)
Img_msk = tf.image.flip_left_right(Img_msk)
if Normalize:
Img, Img_msk = normalize(Img, Img_msk)
return Img, Img_msk
def MyPreprocessing(Image_Path, Mask_Path, resize_img_size = Img_Size):
# Image
Img = tf.io.read_file(Image_Path)
Img = tf.image.decode_jpeg(Img, channels=3)
Img = tf.image.resize(Img, resize_img_size)
Img = tf.cast(Img, tf.float32) / 255.0
# Mask
Img_msk = tf.io.read_file(Mask_Path)
Img_msk = tf.image.decode_jpeg(Img_msk, channels=3)
Img_msk = tf.image.resize(Img_msk, resize_img_size)
Img_msk = Img_msk[:,:,:1]
Img_msk = tf.math.sign(Img_msk)
return Img, Img_msk
def Dataset_Gen(df, subset):
if subset == 'training':
ds = tf.data.Dataset.from_tensor_slices((df["Image_Path"].values, df["Mask_Path"].values))
ds = ds.map(MyPreprocessing, tf.data.AUTOTUNE)
if subset == 'validation':
ds = tf.data.Dataset.from_tensor_slices((df["Image_Path"].values, df["Mask_Path"].values))
ds = ds.map(MyPreprocessing, tf.data.AUTOTUNE)
ds = ds.map(DataAugmentation, tf.data.AUTOTUNE)
return ds
df = Data.iloc[:,-2:].astype(str)
train_df, val_df = train_test_split(df, test_size = 0.2, random_state = 40)
train = Dataset_Gen(train_df, subset = 'training')
val = Dataset_Gen(val_df, subset = 'validation')
del df, train_df, val_df
Batch_Size = 16
train_ds = train.cache().shuffle(Batch_Size).batch(Batch_Size).repeat()
train_ds = train_ds.prefetch(buffer_size=tf.data.AUTOTUNE)
val_ds = val.batch(Batch_Size)
We use a modified U-Net. A U-Net consists of an encoder (downsampler) and decoder (upsampler). We load and modify an exiting MobileNetV2 model from tf.keras.applications
# Model is taken from https://www.tensorflow.org/tutorials/images/segmentation
Img_Size.append(3)
base_model = tf.keras.applications.MobileNetV2(input_shape= Img_Size, include_top=False)
# Use the activations of these layers
layer_names = ['block_1_expand_relu', # 64x64
'block_3_expand_relu', # 32x32
'block_6_expand_relu', # 16x16
'block_13_expand_relu', # 8x8
'block_16_project', # 4x4
]
base_model_outputs = [base_model.get_layer(name).output for name in layer_names]
# Create the feature extraction model
down_stack = tf.keras.Model(inputs=base_model.input, outputs=base_model_outputs)
down_stack.trainable = False
clear_output()
The upsample block is already implemented in TensorFlow Examples in the Pix2pix tutorial.
up_stack = [pix2pix.upsample(512, 3), # 4x4 -> 8x8
pix2pix.upsample(256, 3), # 8x8 -> 16x16
pix2pix.upsample(128, 3), # 16x16 -> 32x32
pix2pix.upsample(64, 3), # 32x32 -> 64x64
]
# This function is taken from https://www.tensorflow.org/tutorials/images/segmentation
def unet_model(output_channels, Img_Size = Img_Size):
inputs = tf.keras.layers.Input(shape= Img_Size)
# Downsampling through the model
skips = down_stack(inputs)
x = skips[-1]
skips = reversed(skips[:-1])
# Upsampling and establishing the skip connections
for up, skip in zip(up_stack, skips):
x = up(x)
concat = tf.keras.layers.Concatenate()
x = concat([x, skip])
# This is the last layer of the model
last = tf.keras.layers.Conv2DTranspose(
output_channels, 3, strides=2, activation='sigmoid',
padding='same') #64x64 -> 128x128
x = last(x)
return tf.keras.Model(inputs=inputs, outputs=x)
Compiling the model
def dice_coef(y_true, y_pred, smooth=1):
'''
Dice coefficient loss function is negative and increasing with epochs.
'''
intersection = tf.keras.backend.sum(y_true * y_pred, axis=[1, 2, 3])
union = tf.keras.backend.sum(y_true, axis=[1,2,3]) + tf.keras.backend.sum(y_pred, axis=[1, 2, 3])
return tf.keras.backend.mean( (2. * intersection + smooth) / (union + smooth), axis=0)
def dice_loss(in_gt, in_pred):
return 1-dice_coef(in_gt, in_pred)
model = unet_model(1)
model.compile(optimizer='adam', loss = dice_loss, metrics=[dice_coef,'binary_accuracy'])
# tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested = False, rankdir= 'LR')
model.summary()
tf.keras.utils.plot_model(model, show_shapes=True)
Model: "model_1"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_2 (InputLayer) [(None, 64, 64, 3)] 0
__________________________________________________________________________________________________
model (Functional) [(None, 32, 32, 96), 1841984 input_2[0][0]
__________________________________________________________________________________________________
sequential (Sequential) (None, 4, 4, 512) 1476608 model[0][4]
__________________________________________________________________________________________________
concatenate (Concatenate) (None, 4, 4, 1088) 0 sequential[0][0]
model[0][3]
__________________________________________________________________________________________________
sequential_1 (Sequential) (None, 8, 8, 256) 2507776 concatenate[0][0]
__________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, 8, 8, 448) 0 sequential_1[0][0]
model[0][2]
__________________________________________________________________________________________________
sequential_2 (Sequential) (None, 16, 16, 128) 516608 concatenate_1[0][0]
__________________________________________________________________________________________________
concatenate_2 (Concatenate) (None, 16, 16, 272) 0 sequential_2[0][0]
model[0][1]
__________________________________________________________________________________________________
sequential_3 (Sequential) (None, 32, 32, 64) 156928 concatenate_2[0][0]
__________________________________________________________________________________________________
concatenate_3 (Concatenate) (None, 32, 32, 160) 0 sequential_3[0][0]
model[0][0]
__________________________________________________________________________________________________
conv2d_transpose_4 (Conv2DTrans (None, 64, 64, 1) 1441 concatenate_3[0][0]
==================================================================================================
Total params: 6,501,345
Trainable params: 4,657,441
Non-trainable params: 1,843,904
__________________________________________________________________________________________________
for images, masks in train_ds.take(1):
for img, mask in zip(images, masks):
sample_image = img
sample_mask = mask
break
def visualize(display_list):
plt.figure(figsize=(15, 15))
title = ['Input Image', 'True Mask', 'Predicted Mask']
for i in range(len(display_list)):
plt.subplot(1, len(display_list), i+1)
plt.title(title[i])
plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))
plt.axis('off')
plt.show()
def show_predictions(sample_image, sample_mask, Img_Size = Img_Size):
pred_mask = model.predict(sample_image[tf.newaxis, ...])
pred_mask = pred_mask.reshape(Img_Size[0], Img_Size[1],1)
visualize([sample_image, sample_mask, pred_mask])
show_predictions(sample_image, sample_mask)
Fitting the model
Batch_Size = 16
IT = 21
STEPS_PER_EPOCH = TRAIN_LENGTH = len(train) // Batch_Size
history = model.fit(train_ds, epochs= IT, steps_per_epoch=STEPS_PER_EPOCH, validation_data=val_ds,
callbacks=[tf.keras.callbacks.EarlyStopping(patience=4, restore_best_weights=True)])
Epoch 1/21 13/13 [==============================] - 5s 220ms/step - loss: 0.5458 - dice_coef: 0.4542 - binary_accuracy: 0.5539 - val_loss: 0.4542 - val_dice_coef: 0.5445 - val_binary_accuracy: 0.5885 Epoch 2/21 13/13 [==============================] - 2s 137ms/step - loss: 0.3365 - dice_coef: 0.6569 - binary_accuracy: 0.8835 - val_loss: 0.2981 - val_dice_coef: 0.7015 - val_binary_accuracy: 0.8297 Epoch 3/21 13/13 [==============================] - 2s 131ms/step - loss: 0.1967 - dice_coef: 0.8013 - binary_accuracy: 0.9412 - val_loss: 0.2654 - val_dice_coef: 0.7350 - val_binary_accuracy: 0.8408 Epoch 4/21 13/13 [==============================] - 2s 131ms/step - loss: 0.1412 - dice_coef: 0.8584 - binary_accuracy: 0.9533 - val_loss: 0.1867 - val_dice_coef: 0.8127 - val_binary_accuracy: 0.9038 Epoch 5/21 13/13 [==============================] - 2s 129ms/step - loss: 0.1162 - dice_coef: 0.8842 - binary_accuracy: 0.9594 - val_loss: 0.1338 - val_dice_coef: 0.8658 - val_binary_accuracy: 0.9384 Epoch 6/21 13/13 [==============================] - 2s 131ms/step - loss: 0.1022 - dice_coef: 0.8968 - binary_accuracy: 0.9621 - val_loss: 0.1080 - val_dice_coef: 0.8917 - val_binary_accuracy: 0.9545 Epoch 7/21 13/13 [==============================] - 2s 133ms/step - loss: 0.0920 - dice_coef: 0.9084 - binary_accuracy: 0.9646 - val_loss: 0.0984 - val_dice_coef: 0.9019 - val_binary_accuracy: 0.9594 Epoch 8/21 13/13 [==============================] - 2s 131ms/step - loss: 0.0839 - dice_coef: 0.9163 - binary_accuracy: 0.9673 - val_loss: 0.0923 - val_dice_coef: 0.9078 - val_binary_accuracy: 0.9625 Epoch 9/21 13/13 [==============================] - 2s 133ms/step - loss: 0.0809 - dice_coef: 0.9189 - binary_accuracy: 0.9673 - val_loss: 0.0912 - val_dice_coef: 0.9098 - val_binary_accuracy: 0.9640 Epoch 10/21 13/13 [==============================] - 2s 139ms/step - loss: 0.0743 - dice_coef: 0.9262 - binary_accuracy: 0.9700 - val_loss: 0.0973 - val_dice_coef: 0.9039 - val_binary_accuracy: 0.9628 Epoch 11/21 13/13 [==============================] - 2s 134ms/step - loss: 0.0724 - dice_coef: 0.9280 - binary_accuracy: 0.9698 - val_loss: 0.1053 - val_dice_coef: 0.8952 - val_binary_accuracy: 0.9604 Epoch 12/21 13/13 [==============================] - 2s 132ms/step - loss: 0.0671 - dice_coef: 0.9333 - binary_accuracy: 0.9730 - val_loss: 0.1027 - val_dice_coef: 0.8983 - val_binary_accuracy: 0.9619 Epoch 13/21 13/13 [==============================] - 2s 136ms/step - loss: 0.0655 - dice_coef: 0.9344 - binary_accuracy: 0.9717 - val_loss: 0.1057 - val_dice_coef: 0.8957 - val_binary_accuracy: 0.9594
def Search_List(Key, List): return [s for s in List if Key in s]
Metrics_Names = dict(zip(model.metrics_names, [x.replace('_',' ').title() for x in model.metrics_names]))
def Table_modify(df, Metrics_Names = Metrics_Names):
df = df.rename(columns = Metrics_Names)
df = df.reindex(sorted(df.columns), axis=1)
df.insert(loc = 0, column = 'Iteration', value = np.arange(0, df.shape[0]), allow_duplicates=False)
return df
Validation_Table = Search_List('val_',history.history.keys())
Train_Table = list(set( history.history.keys()) - set(Validation_Table))
Validation_Table = pd.DataFrame(np.array([history.history[x] for x in Validation_Table]).T, columns = Validation_Table)
Train_Table = pd.DataFrame(np.array([history.history[x] for x in Train_Table]).T, columns = Train_Table)
Validation_Table.columns = [x.replace('val_','') for x in Validation_Table.columns]
Train_Table = Table_modify(Train_Table)
Validation_Table = Table_modify(Validation_Table)
def Plot_history(history, PD, Title = False, metrics_names = list(Metrics_Names.values())):
fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
specs=[[{"type": "scatter"},{"type": "table"}]])
# Left
Colors = ['OrangeRed', 'MidnightBlue', 'purple']
for j in range(len(metrics_names)):
fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history[metrics_names[j]].values,
line=dict(color=Colors[j], width= 1.5), name = metrics_names[j]), 1, 1)
fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
legend_orientation='h')
fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
showgrid=True, gridwidth=1, gridcolor='Lightgray',
showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
fig.update_yaxes(range=[0, PD['yLim']], showgrid=True, gridwidth=1, gridcolor='Lightgray',
showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
# Right
if not PD['Table_Rows'] == None:
ind = np.linspace(0, history.shape[0], PD['Table_Rows'], endpoint = False).round(0).astype(int)
ind = np.append(ind, history.index[-1])
history = history[history.index.isin(ind)]
T = history.copy()
T[metrics_names] = T[metrics_names].applymap(lambda x: '%.4e' % x)
Temp = []
for i in T.columns:
Temp.append(T.loc[:,i].values)
TableColors = PD['TableColors']
fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color=TableColors[0],
fill_color=TableColors[0], align=['center','center'], font=dict(color=TableColors[1], size=12), height=25),
columnwidth = PD['tablecolumnwidth'], cells=dict(values=Temp, line_color=TableColors[0],
fill=dict(color=[TableColors[1], TableColors[1]]),
align=['center', 'center'], font_size=12,height=20)), 1, 2)
if Title != False:
fig.update_layout(plot_bgcolor= 'white',
title={'text': Title, 'x':0.46, 'y':0.94, 'xanchor': 'center', 'yanchor': 'top'},
yaxis_title='Frequency')
fig.show()
PD = dict(Table_Rows = 25, yLim = 1, tablecolumnwidth = [0.3, 0.4, 0.4], TableColors = ['DarkSlateGray','White'])
Plot_history(Train_Table, Title = 'Train Set', PD = PD)
Plot_history(Validation_Table, Title = 'Validation Set', PD = PD)
Removing converted jpg images.
if os.path.exists(NewPath):
shutil.rmtree(NewPath)